from bs4 import BeautifulSoup
from spider import spider


def get_class_details(response):
    data = []
    resp = response.text
    soup = BeautifulSoup(resp, "html.parser")
    div = soup.find('div', class_='explore-categories__container')
    li_list = div.find_all('li', class_='explore-categories__item')
    for item in li_list:
        a = item.find('a')
        url = 'https://gitee.com'+a.get('href')
        flag = a.text.replace('\n', '')
        classify = (flag, url)
        data.append(classify)
    return data

def get_class_in_homepage():
    url = "https://gitee.com/explore/all?order=starred"
    tasks = list()
    param = dict()
    classify = list()
    flag = ['分类']
    task = (url, flag)
    tasks.append(task)
    ret = spider(20, tasks)
    for flag, response in ret:
        if response.status_code == 200:
            # print(response.text)
            classify = get_class_details(response)
    for sheet, url in classify:
        print(sheet)
        print(url)
    print(classify)
    return classify

def get_page_in_homepage(language_list):
    tasks = list()
    page = list()
    for language, link in language_list:
        url = link
        flag = language
        task = (url, flag)
        tasks.append(task)
    ret = spider(20, tasks)
    for flag, response in ret:
        if response.status_code == 200:
            page_num = get_page_details(response)
            page.append(page_num)
    return page

def get_page_details(response):
    data = []
    resp = response.text
    soup = BeautifulSoup(resp, "html.parser")
    div = soup.find('div', class_='column center aligned')
    a_list = div.find_all('a', class_='item')
    # print(len(a_list))
    final_page_text = a_list[-2]
    final_page = final_page_text.text
    # print(final_page)
    data.append(final_page)
    return final_page


if __name__ == "__main__":
    # get_class_in_homepage()
    language_list = [('C', 'https://gitee.com/explore/all?lang=C&order=starred'),
                     ('C++', 'https://gitee.com/explore/all?lang=cpp&order=starred')]
    get_page_in_homepage(language_list)